"""
select+回调+事件循环获取html

使用select完成HTTP请求

核心知识点：
    1.推荐使用selectors库中的select
    2.常用模式：select（poll/epoll）+ 回调 + 事件循环
    3.并发性高
"""

import socket
import time
from urllib.parse import urlparse
from selectors import DefaultSelector, EVENT_READ, EVENT_WRITE

selector = DefaultSelector()

urls = []
stop = False


class Fetcher:
    def __init__(self):
        self.data = b""

    def readable(self, key):
        d = self.client.recv(1024)
        if d:
            self.data += d
        else:
            selector.unregister(key.fd)

            data = self.data.decode("utf8")
            # 通过两个回车换行符去掉头部信息
            html_data = data.split("\r\n\r\n")[1]
            print(html_data)

            # 删除url，防止select在window或者macos下报错
            urls.remove(self.spider_url)
            if not urls:
                global stop
                stop = True

    def connected(self, key):
        """
        连接后的回调函数
        """

        # 注销事件
        selector.unregister(key.fd)
        self.client.send(
            "GET {} HTTP/1.1\r\nHost:{}\r\nConnection:close\r\n\r\n".format(self.path, self.host).encode("utf8"))
        # 注册（可读事件）
        selector.register(self.client.fileno(), EVENT_READ, self.readable)

    def get_url(self, url):
        self.spider_url = url
        # 通过Socket请求html
        url = urlparse(url)
        self.host = url.netloc
        self.path = url.path
        if not self.path:
            self.path = "/"

        # 建立Socket连接
        self.client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

        # 启用非阻塞
        self.client.setblocking(False)

        # connect阻塞不会消耗CPU
        try:
            self.client.connect((self.host, 80))
        except BlockingIOError as e:
            pass

        # 注册
        selector.register(self.client.fileno(), EVENT_WRITE, self.connected)


def event_loop():
    """
    事件循环，不停的请求Socket的状态并调用对应的回调函数。
    """

    # 1.select本身是不支持register模式
    # 2.socket状态变化以后的回调是由程序员来完成的
    while not stop:
        ready = selector.select()
        for key, mask in ready:
            call_back = key.data
            call_back(key)


def use_socket_http():
    start_time = time.time()
    for url in range(1, 10):
        url = "http://shop.projectsedu.com/goods/{}/".format(url)
        urls.append(url)

    for url in urls:
        Fetcher().get_url(url)

    event_loop()
    print(time.time() - start_time)
    # 输出结果：
    """
    {"id":4,"category":{"id":129,"sub_cat":[],"name":"根茎类","code":"gjl","desc":"","category_type":2,"is_tab":false,"add_time":"2017-07-29T18:56:34","parent_category":110},"images":[{"image":"http://shop.projectsedu.com/media/goods/images/47_P_1448946213263.jpg"},{"image":"http://shop.projectsedu.com/media/goods/images/47_P_1448946213157.jpg"}],"goods_sn":"","name":"日本蒜蓉粉丝扇贝270克6只装","click_num":354,"sold_num":0,"fav_num":-1,"goods_num":-15,"market_price":156.0,"shop_price":108.0,"goods_brief":"","goods_desc":"<p><img src=\"/media/goods/images/2_20170719161405_249.jpg\" title=\"\" alt=\"2.jpg\"/></p><p><img src=\"/media/goods/images/2_20170719161414_628.jpg\" title=\"\" alt=\"2.jpg\"/></p><p><img src=\"/media/goods/images/2_20170719161435_381.jpg\" title=\"\" alt=\"2.jpg\"/></p>","ship_free":true,"goods_front_image":"http://shop.projectsedu.com/media/goods/images/47_P_1448946213263.jpg","is_new":false,"is_hot":false,"add_time":"2017-07-31T23:53:53"}
    {"id":3,"category":{"id":124,"sub_cat":[{"id":125,"sub_cat":[],"name":"生菜","code":"sc","desc":"","category_type":3,"is_tab":false,"add_time":"2017-07-29T18:56:34","parent_category":124},{"id":126,"sub_cat":[],"name":"菠菜","code":"bc","desc":"","category_type":3,"is_tab":false,"add_time":"2017-07-29T18:56:34","parent_category":124},{"id":127,"sub_cat":[],"name":"圆椒","code":"yj","desc":"","category_type":3,"is_tab":false,"add_time":"2017-07-29T18:56:34","parent_category":124},{"id":128,"sub_cat":[],"name":"西兰花","code":"xlh","desc":"","category_type":3,"is_tab":false,"add_time":"2017-07-29T18:56:34","parent_category":124}],"name":"叶菜类","code":"ycl","desc":"","category_type":2,"is_tab":false,"add_time":"2017-07-29T18:56:34","parent_category":110},"images":[{"image":"http://shop.projectsedu.com/media/goods/images/7_P_1448945104883.jpg"},{"image":"http://shop.projectsedu.com/media/goods/images/7_P_1448945104734.jpg"}],"goods_sn":"","name":"酣畅家庭菲力牛排10片澳洲生鲜牛肉团购套餐","click_num":298,"sold_num":0,"fav_num":0,"goods_num":-11,"market_price":286.0,"shop_price":238.0,"goods_brief":"","goods_desc":"<p><img src=\"/media/goods/images/2_20170719161405_249.jpg\" title=\"\" alt=\"2.jpg\"/></p><p><img src=\"/media/goods/images/2_20170719161414_628.jpg\" title=\"\" alt=\"2.jpg\"/></p><p><img src=\"/media/goods/images/2_20170719161435_381.jpg\" title=\"\" alt=\"2.jpg\"/></p>","ship_free":true,"goods_front_image":"http://shop.projectsedu.com/media/goods/images/7_P_1448945104883.jpg","is_new":false,"is_hot":false,"add_time":"2017-07-31T23:53:53"}
    {"id":6,"category":{"id":130,"sub_cat":[],"name":"茄果类","code":"qgl","desc":"","category_type":2,"is_tab":false,"add_time":"2017-07-29T18:56:34","parent_category":110},"images":[{"image":"http://shop.projectsedu.com/media/goods/images/4_P_1448945381985.jpg"},{"image":"http://shop.projectsedu.com/media/goods/images/4_P_1448945381013.jpg"}],"goods_sn":"","name":"乌拉圭进口牛肉卷特级肥牛卷","click_num":243,"sold_num":0,"fav_num":1,"goods_num":0,"market_price":90.0,"shop_price":75.0,"goods_brief":"","goods_desc":"<p><img src=\"/media/goods/images/2_20170719161405_249.jpg\" title=\"\" alt=\"2.jpg\"/></p><p><img src=\"/media/goods/images/2_20170719161414_628.jpg\" title=\"\" alt=\"2.jpg\"/></p><p><img src=\"/media/goods/images/2_20170719161435_381.jpg\" title=\"\" alt=\"2.jpg\"/></p>","ship_free":true,"goods_front_image":"http://shop.projectsedu.com/media/goods/images/4_P_1448945381985.jpg","is_new":false,"is_hot":false,"add_time":"2017-07-31T23:53:53"}
    {"id":1,"category":{"id":129,"sub_cat":[],"name":"根茎类","code":"gjl","desc":"","category_type":2,"is_tab":false,"add_time":"2017-07-29T18:56:34","parent_category":110},"images":[{"image":"http://shop.projectsedu.com/media/goods/images/1_P_1449024889889.jpg"},{"image":"http://shop.projectsedu.com/media/goods/images/1_P_1449024889264.jpg"},{"image":"http://shop.projectsedu.com/media/goods/images/1_P_1449024889726.jpg"},{"image":"http://shop.projectsedu.com/media/goods/images/1_P_1449024889018.jpg"},{"image":"http://shop.projectsedu.com/media/goods/images/1_P_1449024889287.jpg"}],"goods_sn":"","name":"新鲜水果甜蜜香脆单果约800克","click_num":1059,"sold_num":0,"fav_num":-1,"goods_num":-45,"market_price":232.0,"shop_price":156.0,"goods_brief":"食用百香果可以增加胃部饱腹感，减少余热量的摄入，还可以吸附胆固醇和胆汁之类有机分子，抑制人体对脂肪的吸收。因此，长期食用有利于改善人体营养吸收结构，降低体内脂肪，塑造健康优美体态。","goods_desc":"<p><img src=\"/media/goods/images/2_20170719161405_249.jpg\" title=\"\" alt=\"2.jpg\"/></p><p><img src=\"/media/goods/images/2_20170719161414_628.jpg\" title=\"\" alt=\"2.jpg\"/></p><p><img src=\"/media/goods/images/2_20170719161435_381.jpg\" title=\"\" alt=\"2.jpg\"/></p>","ship_free":true,"goods_front_image":"http://shop.projectsedu.com/media/goods/images/1_P_1449024889889.jpg","is_new":false,"is_hot":false,"add_time":"2017-07-31T23:53:52"}
    {"id":9,"category":{"id":131,"sub_cat":[],"name":"菌菇类","code":"jgl","desc":"","category_type":2,"is_tab":false,"add_time":"2017-07-29T18:56:34","parent_category":110},"images":[{"image":"http://shop.projectsedu.com/media/goods/images/6_P_1448945167279.jpg"},{"image":"http://shop.projectsedu.com/media/goods/images/6_P_1448945167015.jpg"}],"goods_sn":"","name":"潮香村澳洲进口牛排家庭团购套餐20片","click_num":198,"sold_num":0,"fav_num":0,"goods_num":0,"market_price":239.0,"shop_price":199.0,"goods_brief":"","goods_desc":"<p><img src=\"/media/goods/images/2_20170719161405_249.jpg\" title=\"\" alt=\"2.jpg\"/></p><p><img src=\"/media/goods/images/2_20170719161414_628.jpg\" title=\"\" alt=\"2.jpg\"/></p><p><img src=\"/media/goods/images/2_20170719161435_381.jpg\" title=\"\" alt=\"2.jpg\"/></p>","ship_free":true,"goods_front_image":"http://shop.projectsedu.com/media/goods/images/6_P_1448945167279.jpg","is_new":false,"is_hot":false,"add_time":"2017-07-31T23:53:53"}
    {"id":7,"category":{"id":132,"sub_cat":[],"name":"进口生鲜","code":"jksx","desc":"","category_type":2,"is_tab":false,"add_time":"2017-07-29T18:56:34","parent_category":110},"images":[{"image":"http://shop.projectsedu.com/media/goods/images/8_P_1448945032810.jpg"},{"image":"http://shop.projectsedu.com/media/goods/images/8_P_1448945032646.jpg"}],"goods_sn":"","name":"五星眼肉牛排套餐8片装原味原切生鲜牛肉","click_num":202,"sold_num":0,"fav_num":0,"goods_num":0,"market_price":150.0,"shop_price":125.0,"goods_brief":"","goods_desc":"<p><img src=\"/media/goods/images/2_20170719161405_249.jpg\" title=\"\" alt=\"2.jpg\"/></p><p><img src=\"/media/goods/images/2_20170719161414_628.jpg\" title=\"\" alt=\"2.jpg\"/></p><p><img src=\"/media/goods/images/2_20170719161435_381.jpg\" title=\"\" alt=\"2.jpg\"/></p>","ship_free":true,"goods_front_image":"http://shop.projectsedu.com/media/goods/images/8_P_1448945032810.jpg","is_new":false,"is_hot":false,"add_time":"2017-07-31T23:53:53"}
    {"id":8,"category":{"id":116,"sub_cat":[{"id":117,"sub_cat":[],"name":"参鲍","code":"cb","desc":"","category_type":3,"is_tab":false,"add_time":"2017-07-29T18:56:33","parent_category":116},{"id":118,"sub_cat":[],"name":"鱼","code":"yu","desc":"","category_type":3,"is_tab":false,"add_time":"2017-07-29T18:56:33","parent_category":116},{"id":119,"sub_cat":[],"name":"虾","code":"xia","desc":"","category_type":3,"is_tab":false,"add_time":"2017-07-29T18:56:33","parent_category":116},{"id":120,"sub_cat":[],"name":"蟹/贝","code":"xb","desc":"","category_type":3,"is_tab":false,"add_time":"2017-07-29T18:56:33","parent_category":116}],"name":"海鲜水产","code":"hxsc","desc":"","category_type":2,"is_tab":false,"add_time":"2017-07-29T18:56:33","parent_category":110},"images":[{"image":"http://shop.projectsedu.com/media/goods/images/11_P_1448944388277.jpg"},{"image":"http://shop.projectsedu.com/media/goods/images/11_P_1448944388034.jpg"},{"image":"http://shop.projectsedu.com/media/goods/images/11_P_1448944388201.jpg"}],"goods_sn":"","name":"澳洲进口120天谷饲牛仔骨4份原味生鲜","click_num":203,"sold_num":0,"fav_num":0,"goods_num":0,"market_price":31.0,"shop_price":26.0,"goods_brief":"","goods_desc":"<p><img src=\"/media/goods/images/2_20170719161405_249.jpg\" title=\"\" alt=\"2.jpg\"/></p><p><img src=\"/media/goods/images/2_20170719161414_628.jpg\" title=\"\" alt=\"2.jpg\"/></p><p><img src=\"/media/goods/images/2_20170719161435_381.jpg\" title=\"\" alt=\"2.jpg\"/></p>","ship_free":true,"goods_front_image":"http://shop.projectsedu.com/media/goods/images/11_P_1448944388277.jpg","is_new":false,"is_hot":false,"add_time":"2017-07-31T23:53:53"}
    {"id":5,"category":{"id":116,"sub_cat":[{"id":117,"sub_cat":[],"name":"参鲍","code":"cb","desc":"","category_type":3,"is_tab":false,"add_time":"2017-07-29T18:56:33","parent_category":116},{"id":118,"sub_cat":[],"name":"鱼","code":"yu","desc":"","category_type":3,"is_tab":false,"add_time":"2017-07-29T18:56:33","parent_category":116},{"id":119,"sub_cat":[],"name":"虾","code":"xia","desc":"","category_type":3,"is_tab":false,"add_time":"2017-07-29T18:56:33","parent_category":116},{"id":120,"sub_cat":[],"name":"蟹/贝","code":"xb","desc":"","category_type":3,"is_tab":false,"add_time":"2017-07-29T18:56:33","parent_category":116}],"name":"海鲜水产","code":"hxsc","desc":"","category_type":2,"is_tab":false,"add_time":"2017-07-29T18:56:33","parent_category":110},"images":[{"image":"http://shop.projectsedu.com/media/goods/images/10_P_1448944572085.jpg"},{"image":"http://shop.projectsedu.com/media/goods/images/10_P_1448944572532.jpg"},{"image":"http://shop.projectsedu.com/media/goods/images/10_P_1448944572872.jpg"}],"goods_sn":"","name":"内蒙新鲜牛肉1斤清真生鲜牛肉火锅食材","click_num":248,"sold_num":0,"fav_num":0,"goods_num":0,"market_price":106.0,"shop_price":88.0,"goods_brief":"","goods_desc":"<p><img src=\"/media/goods/images/2_20170719161405_249.jpg\" title=\"\" alt=\"2.jpg\"/></p><p><img src=\"/media/goods/images/2_20170719161414_628.jpg\" title=\"\" alt=\"2.jpg\"/></p><p><img src=\"/media/goods/images/2_20170719161435_381.jpg\" title=\"\" alt=\"2.jpg\"/></p>","ship_free":true,"goods_front_image":"http://shop.projectsedu.com/media/goods/images/10_P_1448944572085.jpg","is_new":false,"is_hot":false,"add_time":"2017-07-31T23:53:53"}
    {"id":2,"category":{"id":116,"sub_cat":[{"id":117,"sub_cat":[],"name":"参鲍","code":"cb","desc":"","category_type":3,"is_tab":false,"add_time":"2017-07-29T18:56:33","parent_category":116},{"id":118,"sub_cat":[],"name":"鱼","code":"yu","desc":"","category_type":3,"is_tab":false,"add_time":"2017-07-29T18:56:33","parent_category":116},{"id":119,"sub_cat":[],"name":"虾","code":"xia","desc":"","category_type":3,"is_tab":false,"add_time":"2017-07-29T18:56:33","parent_category":116},{"id":120,"sub_cat":[],"name":"蟹/贝","code":"xb","desc":"","category_type":3,"is_tab":false,"add_time":"2017-07-29T18:56:33","parent_category":116}],"name":"海鲜水产","code":"hxsc","desc":"","category_type":2,"is_tab":false,"add_time":"2017-07-29T18:56:33","parent_category":110},"images":[{"image":"http://shop.projectsedu.com/media/goods/images/2_P_1448945810202.jpg"},{"image":"http://shop.projectsedu.com/media/15_P_1448947257324.jpg"},{"image":"http://shop.projectsedu.com/media/2_20170719161405_249.jpg"},{"image":"http://shop.projectsedu.com/media/9_P_1448944791617.jpg"}],"goods_sn":"sssss","name":"田然牛肉大黄瓜条生鲜牛肉冷冻真空黄牛","click_num":641,"sold_num":100,"fav_num":1,"goods_num":-21,"market_price":106.0,"shop_price":88.0,"goods_brief":"前腿+后腿+羊排共8斤，原生态大山放牧羊羔，曾经的皇室贡品，央视推荐，2005年北京招待全球财金首脑。五层专用包装箱+真空包装+冰袋+保鲜箱+顺丰冷链发货，路途保质期8天","goods_desc":"<p><img src=\"/media/goods/images/2_20170719161405_249.jpg\" title=\"\" alt=\"2.jpg\"/> </p><p><img src=\"/media/goods/images/2_20170719161414_628.jpg\" title=\"\" alt=\"2.jpg\"/> </p><p><img src=\"/media/goods/images/2_20170719161435_381.jpg\" title=\"\" alt=\"2.jpg\"/> </p>","ship_free":true,"goods_front_image":"http://shop.projectsedu.com/media/goods/images/2_P_1448945810202.jpg","is_new":false,"is_hot":false,"add_time":"2017-07-31T23:53:00"}
    0.5957272052764893
    """


if __name__ == '__main__':
    use_socket_http()
